package com.project.oj.recommend;

import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary;
import com.hankcs.hanlp.seg.common.Term;

import java.util.*;
import java.util.stream.Collectors;

public class TextPreprocessor {
    private static final Set<String> ALLOWED_TAGS = new HashSet<>(Arrays.asList("n", "v", "vd", "vn", "a", "d"));

    public List<String> processText(String text) {
        List<Term> termList = HanLP.segment(text);
        CoreStopWordDictionary.apply(termList);
        Set<String> uniqueWords = termList.stream()
                .filter(term -> ALLOWED_TAGS.contains(term.nature.toString()))
                .map(term -> term.word)
                .collect(Collectors.toSet());
        return new ArrayList<>(uniqueWords);
    }
}
